﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.Diagnostics;
using System.Text.RegularExpressions;
using System.Web;

namespace WebFetch
{
    public struct PageState
    {
        public int offset;
        public bool isGetPageEncoding;
        public int length;
    }
    class Program
    {
        static void Main(string[] args)
        {
            // used to build entire input
            StringBuilder sb = new StringBuilder();

            //List<string> cUrl = new List<string>();
            //List<string> forbitURL = new List<string>();

            //List<string> urlString = new List<string>();
            //List<string> urlList = new List<string>();

            //List<int> fileHash = new List<int>();

            //ParserState state = ParserState.Normal;
            EngineState es = new EngineState();

            // urlString.Add("http://sourceforge.net/");
            // urlString.Add("http://www.nytimes.com");

            //string realURL = urlString[0];
            PageState pageState = new PageState();
            pageState.offset = 0;

            Recursion(new URL(es.UrlString[0]), ref es);

            #region comment
            //TextWriter tw = new StreamWriter(path, false, Encoding.GetEncoding(mainCharset));

            //TextWriter tw = new StreamWriter(@"C:\workspace\FileName.txt", false, Encoding.GetEncoding(mainCharset));
            //tw.Write(temp);


            //foreach (string s in urlList)
            //{
            //    tw.WriteLine(s);
            //}
            //tw.Close();

            //Process notePad = new Process();

            //notePad.StartInfo.FileName = "notepad.exe";
            //notePad.StartInfo.Arguments = path;

            //notePad.Start();

            //System.Diagnostics.Process proc = new System.Diagnostics.Process();
            //proc.EnableRaisingEvents = false;
            //proc.StartInfo.FileName = "c:\\workspace\\firebird\\1.bat";
            //proc.Start();
            // print out page source
            //Console.WriteLine(sb.ToString());
            #endregion
        }

        public static void Recursion(URL url, ref EngineState es)
        {
            #region Current Domain

            es.CurrentDomain = url.HostName;

            #endregion

            if (HtmlParserHelper.GetUrlLayer(url) > 4)
                return;
            if (es.ForbitURL.Contains(url.HostName))
                return;
            if (es.CUrl.Contains(url.Url))
                return;
            es.CUrl.Add(url.Url);


            string realURL = url.Url;

            List<string> urlList = new List<string>();

            string temp = "";
            string baseTagHref = "";
            State state = HtmlParserHelper.getHtmlFile(realURL, es.MainCharset, ref temp);
            if (state == State.FAIL)
                return;

            if (es.FileHash.Contains(temp.GetHashCode()))
                return;
            es.FileHash.Add(temp.GetHashCode());

            int i = 0;
            string path = HtmlParserHelper.makeUrlDir("D:\\awork\\", url, temp.GetHashCode());
            TextWriter tw = new StreamWriter(path, false, Encoding.GetEncoding(es.MainCharset));
            Console.WriteLine(realURL);

            while (i < temp.Length)
            {
                int length = i;
                switch (temp[i])
                {
                    case '<':
                        Tags tag = new Tags();
                        while (length < temp.Length && temp[length] != ' ' && temp[length] != '>') length++;
                        tag.TagsName = temp.Substring(i + 1, length - i - 1);

                        //switch (tag.Type)
                        //{
                        //    case TagsType.DoubleCloseTags:
                        //        getTagContent(temp.Substring(i + 1, length - i - 1), length + 1);
                        //        break;
                        //}
                        string hrefAttribute = "";
                        switch (tag.TagsName)
                        {
                            case "base":
                                baseTagHref = HtmlParserHelper.getProperty(temp, length, "href");
                                break;
                            case "a":
                            case "A":
                                hrefAttribute = HtmlParserHelper.getProperty(temp, length, "href");
                                if (hrefAttribute != null
                                    && hrefAttribute != ""
                                    && hrefAttribute.Contains("javascript") == false
                                    && hrefAttribute.Contains("mailto") == false)
                                    urlList.Add(hrefAttribute);
                                break;
                            case "img":
                            //hrefPropery = getProperty(temp, length, "src");
                            //break;
                            case "b":
                            case "B":
                            case "H1":
                            case "h1":
                            case "p":
                            case "P":
                            case "pre":
                            case "/a":
                            case "/b":
                            case "/H1":
                            case "H2":
                            case "/H2":
                            case "h2":
                            case "/h2":
                            case "H3":
                            case "/H3":
                            case "h3":
                            case "/h3":
                            case "/pre":
                            case "div":
                            case "/div":
                            case "td":
                            case "/td":
                            case "tr":
                            case "/tr":
                            case "span":
                            case "/span":
                            case "strong":
                            case "/strong":
                            case "li":
                            case "/li":

                                //length = HtmlParserHelper.moveToThisTagEnd(temp, length);
                                ////Console.Write(temp.Substring(length+1,moveToNextTagStart(temp,length)-length-1)+" ");
                                //string content = temp.Substring(length + 1, HtmlParserHelper.moveToNextTagStart(temp, length) - length - 1) + " ";
                                //Regex r = new Regex("^[ \t\r\n]*$");
                                //if (r.IsMatch(content) == false)
                                //    tw.WriteLine(content);
                                break;

                            case "meta":
                                string charset = HtmlParserHelper.getProperty(temp, length, "charset");

                                if (charset != null && charset != es.MainCharset)
                                {
                                    State stateLocal = HtmlParserHelper.getHtmlFile(realURL, charset, ref temp);

                                    if (stateLocal == State.FAIL)
                                        return;

                                    es.MainCharset = charset;
                                    tw.Close();
                                    tw = new StreamWriter(path, false, Encoding.GetEncoding(es.MainCharset));
                                }
                                //Console.Write(temp);
                                break;
                        }
                        i = length;
                        break;
                }
                i++;
            }

            tw.Write(temp);
            tw.Close();

            /*
1.html files contains lots of files
2.make this url to url Stack.
+-------+    URL1                  
|       |    URL2                  
| html  | -> URL3->URLStack  
|       |    URL4     |            
+-------+             |   
                      |   
-->                   |         
pop from url stack and|into main url stack.
+--+--+--+--+---+--+--V--+--+--+
| main url stack|              |
+--+--+--+--+---+--+--+--+--+--+    
             * */

            /*
Database Stack
This Stack contains all url we visits, 
it need to save into database. 
             * */
            Stack<string> database = new Stack<string>();

            Stack<string> validateLink = new Stack<string>();

            //main url queue
            Queue<string> mainQueue = new Queue<string>();

            foreach (string urlchild in urlList)
            {
                if (mainQueue.Contains(urlchild) == false)
                {
                    if (urlchild != null
                        && urlchild != ""
                        && urlchild != "/"
                        && urlchild.StartsWith("#") == false
                        )
                    {
                        #region check domain
                        if (urlchild.StartsWith("http://") == false)
                        {

                        }
                        else if ((new URL(urlchild)).DomainName != HtmlParserHelper.GetDomainName(es.CurrentDomain))
                        {
                            InsertBackupURL(ref es, urlchild);
                            continue;
                        }
                        #endregion
                        mainQueue.Enqueue(urlchild);
                    }
                }
            }
            TextWriter queuewriter = new StreamWriter(@"D:\\awork\\queuewriter.txt", true, Encoding.GetEncoding("UTF-8"));
            try
            {
                while (mainQueue.Count > 0)
                {
                    string childUrlTemp = mainQueue.Dequeue();
                    queuewriter.WriteLine(childUrlTemp);
                    queuewriter.Flush();
                    /*
                     *  +-----+----+-----+-----+
                        |     |    |     |     |
                        +-----+----+-----+-----+
                          |                     
                          |
                          V
                        Check whether    
                        the url is belong to domain.
                     * */

                    database.Push(childUrlTemp);
                    Queue<string> childUrlStack = new Queue<string>();
                    if (childUrlTemp.Contains("mailto:") == true)
                    {
                        Console.WriteLine(childUrlTemp + "is a email address");
                        continue;
                    }

                    if (childUrlTemp.StartsWith("http://") == false)
                    {
                        childUrlStack = getStack(new URL(HtmlParserHelper.ConcatURL(url.CurrentPath, childUrlTemp)),ref es);
                    }
                    else
                    {
                        childUrlStack = getStack(new URL(childUrlTemp),ref es);
                    }

                    //pop up childUrlStack, check the url validate
                    while (childUrlStack.Count > 0)
                    {
                        string s = childUrlStack.Dequeue();
                        if (mainQueue.Contains(s) == false && validateLink.Contains(s) == false && database.Contains(s) == false)
                        {
                            #region check domain
                            if (s.StartsWith("http://") == false)
                            {

                            }
                            else if ((new URL(s)).DomainName != HtmlParserHelper.GetDomainName(es.CurrentDomain))
                            {
                                InsertBackupURL(ref es, s);
                                continue;
                            }
                            #endregion
                            mainQueue.Enqueue(s);
                        }
                    }
                }
                queuewriter.Close();
            }
            catch (System.Exception ex)
            {
                queuewriter.Close();
                TextWriter logwriter = new StreamWriter(@"D:\\awork\\urllog.txt", true, Encoding.GetEncoding("UTF-8"));
                logwriter.WriteLine(ex.Message);
                logwriter.WriteLine(ex.StackTrace);
                logwriter.WriteLine(ex.Source);
                logwriter.WriteLine(ex.InnerException);
                logwriter.Flush();
                logwriter.WriteLine("mainQueue================================");
                foreach (string s in mainQueue)
                {
                    logwriter.WriteLine(s);
                    logwriter.Flush();
                }
                logwriter.WriteLine("mainQueue end============================");
                logwriter.Close();
            }
        }

        private static void InsertBackupURL(ref EngineState es, string backupURL)
        {
            if (backupURL != null)
                es.BackupURL.Add(backupURL);
        }

        public static Queue<string> getStack(URL url, ref EngineState es)
        {
            string realURL = url.Url;

            List<string> urlList = new List<string>();

            string temp = "";
            string baseTagHref = "";
            State state = HtmlParserHelper.getHtmlFile(realURL, es.MainCharset, ref temp);
            if (state == State.FAIL)
                return new Queue<string>();

            int i = 0;
            //TextWriter pathwriter = new StreamWriter(@"D:\\awork\\path.txt", true, Encoding.GetEncoding("UTF-8"));
            //pathwriter.WriteLine(DateTime.Now+"---"+url.CurrentPath+"---"+url.FileName);
            //pathwriter.Flush();
            //pathwriter.Close();
            string path = HtmlParserHelper.makeUrlDir("D:\\awork\\", url, temp.GetHashCode());
            TextWriter tw = new StreamWriter(path, false, Encoding.GetEncoding(es.MainCharset));
            Console.WriteLine(realURL);

            while (i < temp.Length)
            {
                int length = i;
                switch (temp[i])
                {
                    case '<':
                        Tags tag = new Tags();
                        while (temp[length] != ' ' && temp[length] != '>') length++; 
                        tag.TagsName = temp.Substring(i + 1, length - i - 1);

                        //switch (tag.Type)
                        //{
                        //    case TagsType.DoubleCloseTags:
                        //        getTagContent(temp.Substring(i + 1, length - i - 1), length + 1);
                        //        break;
                        //}
                        string hrefAttribute = "";
                        switch (tag.TagsName)
                        {
                            case "base":
                                baseTagHref = HtmlParserHelper.getProperty(temp, length, "href");
                                break;
                            case "a":
                            case "A":
                                hrefAttribute = HtmlParserHelper.getProperty(temp, length, "href");
                                if (hrefAttribute != null && hrefAttribute != "" && hrefAttribute.Contains("javascript") == false)
                                    urlList.Add(hrefAttribute);
                                break;
                            case "img":
                            //hrefPropery = getProperty(temp, length, "src");
                            //break;
                            case "b":
                            case "B":
                            case "H1":
                            case "h1":
                            case "p":
                            case "P":
                            case "pre":
                            case "/a":
                            case "/b":
                            case "/H1":
                            case "H2":
                            case "/H2":
                            case "h2":
                            case "/h2":
                            case "H3":
                            case "/H3":
                            case "h3":
                            case "/h3":
                            case "/pre":
                            case "div":
                            case "/div":
                            case "td":
                            case "/td":
                            case "tr":
                            case "/tr":
                            case "span":
                            case "/span":
                            case "strong":
                            case "/strong":
                            case "li":
                            case "/li":

                                //length = HtmlParserHelper.moveToThisTagEnd(temp, length);
                                ////Console.Write(temp.Substring(length+1,moveToNextTagStart(temp,length)-length-1)+" ");
                                //string content = temp.Substring(length + 1, HtmlParserHelper.moveToNextTagStart(temp, length) - length - 1) + " ";
                                //Regex r = new Regex("^[ \t\r\n]*$");
                                //if (r.IsMatch(content) == false)
                                //    tw.WriteLine(content);
                                break;

                            case "meta":
                                string charset = HtmlParserHelper.getProperty(temp, length, "charset");

                                
                                if (charset != null && charset != es.MainCharset)
                                {
                                    es.MainCharset = charset;
                                    State stateLocal = HtmlParserHelper.getHtmlFile(realURL, charset, ref temp);

                                    if (stateLocal == State.FAIL)
                                        return new Queue<string>();

                                    tw.Close();
                                    tw = new StreamWriter(path, false, Encoding.GetEncoding(es.MainCharset));
                                }
                                //Console.Write(temp);
                                break;
                        }
                        i = length;
                        break;
                }
                i++;
            }

            tw.Write(temp);
            tw.Close();

            Queue<string> s = new Queue<string>();
            foreach (string urlchild in urlList)
            {
                if (urlchild != null && urlchild != "" && urlchild != "/" && urlchild.StartsWith("#") == false)
                {
                    if (urlchild.StartsWith("http://") == false)
                    {
                        s.Enqueue(HtmlParserHelper.RelateURLToAbsolute(HtmlParserHelper.ConcatURL(url.CurrentPath,urlchild)));
                    }
                    else
                        s.Enqueue(HtmlParserHelper.RelateURLToAbsolute(urlchild));
                }
            }
            return s;
        }
    }
}
